{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.9 64-bit ('DataAnalysis': conda)",
   "metadata": {
    "interpreter": {
     "hash": "dcf060ad4b7fa10e323b7c0259734b0cdb630d87fa5c1539ce2c08c5bd388944"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import brown\n",
    "from nltk.corpus import names\n",
    "\n",
    "from tools import show_subtitle"
   ]
  },
  {
   "source": [
    "# Chap6 学习分类文本\n",
    "\n",
    "学习目标：\n",
    "\n",
    "1.  识别出语言数据中可以用于分类的特征\n",
    "2.  构建用于自动执行语言处理任务的语言模型\n",
    "3.  从语言模型中学习与语言相关的知识"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## 6.1 有监督分类\n",
    "\n",
    "-   分类是为了给定的输入选择正确的类标签。\n",
    "-   监督式分类：建立在训练语料基础之上的分类"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### 6.1.1 性别鉴定"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'last_letter': 'k'}"
      ]
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "# 以名字的最后一个字母为特征\n",
    "def gender_features(word):\n",
    "    return {'last_letter': word[-1]}\n",
    "\n",
    "\n",
    "gender_features('Shrek')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 原始数据集合\n",
    "labeled_names = (\n",
    "        [\n",
    "                (name, 'male')\n",
    "                for name in names.words('male.txt')\n",
    "        ]\n",
    "        +\n",
    "        [\n",
    "                (name, 'female')\n",
    "                for name in names.words('female.txt')\n",
    "        ]\n",
    ")\n",
    "# 乱序排序数据集\n",
    "random.shuffle(labeled_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 特征数据集合\n",
    "feature_sets = [\n",
    "        (gender_features(n), gender)\n",
    "        for (n, gender) in labeled_names\n",
    "]\n",
    "\n",
    "# 训练数据集合 和 测试数据集合\n",
    "train_set, test_set = feature_sets[500:], feature_sets[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "NLTK-NB 分类器性能评估=  0.772\n--------------- >Neo's feature< ---------------\n{'last_letter': 'o'}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'last_letter': 'y'}\nTrinity is female\n--------------- >信息量大的特征< ---------------\nMost Informative Features\n             last_letter = 'a'            female : male   =     33.0 : 1.0\n             last_letter = 'k'              male : female =     31.7 : 1.0\n             last_letter = 'v'              male : female =     18.6 : 1.0\n             last_letter = 'f'              male : female =     14.6 : 1.0\n             last_letter = 'p'              male : female =     12.5 : 1.0\n"
     ]
    }
   ],
   "source": [
    "# 使用 NLTK 的 朴素贝叶斯 分类器进行分类训练\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "print(\"NLTK-NB 分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "\n",
    "show_subtitle(\"Neo's feature\")\n",
    "print(gender_features('Neo'))\n",
    "print(\"Neo is\", classifier.classify(gender_features('Neo')))\n",
    "show_subtitle(\"Trinity's feature\")\n",
    "print(gender_features('Trinity'))\n",
    "print(\"Trinity is\", classifier.classify(gender_features('Trinity')))\n",
    "\n",
    "show_subtitle(\"信息量大的特征\")\n",
    "classifier.show_most_informative_features(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.73\n--------------- >Neo's feature< ---------------\n{'last_letter': 'o'}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'last_letter': 'y'}\nTrinity is female\n"
     ]
    }
   ],
   "source": [
    "# 使用 Scikit-Learn 的 GaussNB 朴素贝叶斯 分类器进行分类\n",
    "import numpy as np\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from nltk.classify.scikitlearn import SklearnClassifier\n",
    "classifier=SklearnClassifier(GaussianNB(), sparse=False).train(train_set)\n",
    "print(\"Scikit-GaussNB 分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "\n",
    "show_subtitle(\"Neo's feature\")\n",
    "print(gender_features('Neo'))\n",
    "print(\"Neo is\", classifier.classify(gender_features('Neo')))\n",
    "show_subtitle(\"Trinity's feature\")\n",
    "print(gender_features('Trinity'))\n",
    "print(\"Trinity is\", classifier.classify(gender_features('Trinity')))\n",
    "\n",
    "# 信息量大的特征\n",
    "# 'SklearnClassifier' object has no attribute 'show_most_informative_features'\n",
    "# classifier.show_most_informative_features(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gender_classifier(features):\n",
    "    # 特征数据集合\n",
    "    feature_sets = [\n",
    "            (features(n), gender)\n",
    "            for (n, gender) in labeled_names\n",
    "    ]\n",
    "\n",
    "    # 训练数据集合 和 测试数据集合\n",
    "    train_set, test_set = feature_sets[500:], feature_sets[:500]\n",
    "\n",
    "    # 使用 Scikit-Learn 的 GaussNB 朴素贝叶斯 分类器进行分类\n",
    "    from sklearn.naive_bayes import GaussianNB\n",
    "    from nltk.classify.scikitlearn import SklearnClassifier\n",
    "\n",
    "    classifier = SklearnClassifier(GaussianNB(), sparse=False).train(train_set)\n",
    "    print(\"Scikit-GaussNB 分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "\n",
    "\n",
    "    # 使用 NLTK 的 朴素贝叶斯 分类器进行分类训练\n",
    "    classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "    print(\"NLTK-NB 分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "\n",
    "    # 信息量大的特征（发现特征信息量比较小，说明这个特征效果不好）\n",
    "    show_subtitle(\"信息量大的特征\")\n",
    "    classifier.show_most_informative_features(5)\n",
    "\n",
    "    show_subtitle(\"Neo's feature\")\n",
    "    print(features('Neo'))\n",
    "    print(\"Neo is\", classifier.classify(features('Neo')))\n",
    "    show_subtitle(\"Trinity's feature\")\n",
    "    print(features('Trinity'))\n",
    "    print(\"Trinity is\", classifier.classify(features('Trinity')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.634\nNLTK-NB 分类器性能评估=  0.64\n--------------- >信息量大的特征< ---------------\nMost Informative Features\n             name length = 2                male : female =      2.1 : 1.0\n             name length = 3                male : female =      2.0 : 1.0\n             name length = 15               male : female =      1.7 : 1.0\n             name length = 9              female : male   =      1.4 : 1.0\n             name length = 10             female : male   =      1.3 : 1.0\n--------------- >Neo's feature< ---------------\n{'name length': 3}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'name length': 7}\nTrinity is female\n"
     ]
    }
   ],
   "source": [
    "# 性别鉴定：以名字的长度为特征\n",
    "# 等价的 Lambda 函数\n",
    "# gender_classifier(lambda word: {'name length': len(word)})\n",
    "def gender_features(word):\n",
    "    return {'name length': len(word)}\n",
    "\n",
    "\n",
    "gender_classifier(gender_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在处理大型语料库时，构建包含所有实例特征的单独链表会占用大量的内存\n",
    "# apply_features 返回一个链表，但是不会在内存中存储所有特征集的对象\n",
    "from nltk.classify import apply_features\n",
    "\n",
    "train_set = apply_features(gender_features, labeled_names[500:])\n",
    "test_set = apply_features(gender_features, labeled_names[:500])"
   ]
  },
  {
   "source": [
    "### 6.1.2 选择正确的特征"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.732\nNLTK-NB 分类器性能评估=  0.768\n--------------- >信息量大的特征< ---------------\nMost Informative Features\n             last_letter = 'a'            female : male   =     33.0 : 1.0\n             last_letter = 'k'              male : female =     31.7 : 1.0\n             last_letter = 'v'              male : female =     18.6 : 1.0\n             last_letter = 'f'              male : female =     14.6 : 1.0\n             last_letter = 'p'              male : female =     12.5 : 1.0\n--------------- >Neo's feature< ---------------\n{'first_letter': 'n', 'last_letter': 'o'}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'first_letter': 't', 'last_letter': 'y'}\nTrinity is male\n"
     ]
    }
   ],
   "source": [
    "# 两个特征可以少量地增加了准确率（0.776>0.774)\n",
    "def gender_features(name):\n",
    "    features = {'first_letter': name[0].lower(), 'last_letter': name[-1].lower()}\n",
    "    return features\n",
    "\n",
    "\n",
    "gender_classifier(gender_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.744\n",
      "NLTK-NB 分类器性能评估=  0.782\n",
      "--------------- >信息量大的特征< ---------------\n",
      "Most Informative Features\n",
      "             last_letter = 'a'            female : male   =     33.0 : 1.0\n",
      "             last_letter = 'k'              male : female =     31.7 : 1.0\n",
      "             last_letter = 'v'              male : female =     18.6 : 1.0\n",
      "             last_letter = 'f'              male : female =     14.6 : 1.0\n",
      "             last_letter = 'p'              male : female =     12.5 : 1.0\n",
      "--------------- >Neo's feature< ---------------\n",
      "{'first_letter': 'n', 'last_letter': 'o', 'count(a)': 0, 'has(a)': False, 'count(b)': 0, 'has(b)': False, 'count(c)': 0, 'has(c)': False, 'count(d)': 0, 'has(d)': False, 'count(e)': 1, 'has(e)': True, 'count(f)': 0, 'has(f)': False, 'count(g)': 0, 'has(g)': False, 'count(h)': 0, 'has(h)': False, 'count(i)': 0, 'has(i)': False, 'count(j)': 0, 'has(j)': False, 'count(k)': 0, 'has(k)': False, 'count(l)': 0, 'has(l)': False, 'count(m)': 0, 'has(m)': False, 'count(n)': 1, 'has(n)': True, 'count(o)': 1, 'has(o)': True, 'count(p)': 0, 'has(p)': False, 'count(q)': 0, 'has(q)': False, 'count(r)': 0, 'has(r)': False, 'count(s)': 0, 'has(s)': False, 'count(t)': 0, 'has(t)': False, 'count(u)': 0, 'has(u)': False, 'count(v)': 0, 'has(v)': False, 'count(w)': 0, 'has(w)': False, 'count(x)': 0, 'has(x)': False, 'count(y)': 0, 'has(y)': False, 'count(z)': 0, 'has(z)': False}\n",
      "Neo is male\n",
      "--------------- >Trinity's feature< ---------------\n",
      "{'first_letter': 't', 'last_letter': 'y', 'count(a)': 0, 'has(a)': False, 'count(b)': 0, 'has(b)': False, 'count(c)': 0, 'has(c)': False, 'count(d)': 0, 'has(d)': False, 'count(e)': 0, 'has(e)': False, 'count(f)': 0, 'has(f)': False, 'count(g)': 0, 'has(g)': False, 'count(h)': 0, 'has(h)': False, 'count(i)': 2, 'has(i)': True, 'count(j)': 0, 'has(j)': False, 'count(k)': 0, 'has(k)': False, 'count(l)': 0, 'has(l)': False, 'count(m)': 0, 'has(m)': False, 'count(n)': 1, 'has(n)': True, 'count(o)': 0, 'has(o)': False, 'count(p)': 0, 'has(p)': False, 'count(q)': 0, 'has(q)': False, 'count(r)': 1, 'has(r)': True, 'count(s)': 0, 'has(s)': False, 'count(t)': 2, 'has(t)': True, 'count(u)': 0, 'has(u)': False, 'count(v)': 0, 'has(v)': False, 'count(w)': 0, 'has(w)': False, 'count(x)': 0, 'has(x)': False, 'count(y)': 1, 'has(y)': True, 'count(z)': 0, 'has(z)': False}\n",
      "Trinity is female\n"
     ]
    }
   ],
   "source": [
    "# 复杂的特征集，导致过拟合，反而降低了准确率\n",
    "def gender_features(name):\n",
    "    # 特征集合\n",
    "    features = {'first_letter': name[0].lower(), 'last_letter': name[-1].lower()}\n",
    "    for letter in 'abcdefghijklmnopqrstuvwxyz':\n",
    "        features['count({})'.format(letter)] = name.lower().count(letter)\n",
    "        features['has({})'.format(letter)] = (letter in name.lower())\n",
    "    return features\n",
    "\n",
    "\n",
    "gender_classifier(gender_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "开发测试集的精度=  0.768\n测试集的精度=  0.782\n"
     ]
    }
   ],
   "source": [
    "# 开发测试集的作用\n",
    "# 训练集：训练模型；\n",
    "# 开发测试集：执行错误分析；\n",
    "# 测试集：系统的最终评估。\n",
    "train_names = labeled_names[1500:]\n",
    "devtest_names = labeled_names[500:1500]\n",
    "test_names = labeled_names[:500]\n",
    "\n",
    "train_set = [(gender_features(n), gender) for (n, gender) in train_names]\n",
    "devtest_set = [(gender_features(n), gender) for (n, gender) in devtest_names]\n",
    "test_set = [(gender_features(n), gender) for (n, gender) in test_names]\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "print(\"开发测试集的精度= \", nltk.classify.accuracy(classifier, devtest_set))\n",
    "print(\"测试集的精度= \", nltk.classify.accuracy(classifier, test_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "correct=female   guess=male     name=Bert                          \ncorrect=female   guess=male     name=Birgit                        \ncorrect=female   guess=male     name=Blondy                        \ncorrect=female   guess=male     name=Ginnifer                      \ncorrect=female   guess=male     name=Gretal                        \ncorrect=female   guess=male     name=Hephzibah                     \ncorrect=female   guess=male     name=Lory                          \ncorrect=female   guess=male     name=Mab                           \ncorrect=female   guess=male     name=Patsy                         \ncorrect=female   guess=male     name=Tammy                         \ncorrect=female   guess=male     name=Tuesday                       \ncorrect=male     guess=female   name=Angel                         \ncorrect=male     guess=female   name=Bobbie                        \ncorrect=male     guess=female   name=Earl                          \ncorrect=male     guess=female   name=Graeme                        \ncorrect=male     guess=female   name=Kalvin                        \ncorrect=male     guess=female   name=Mikhail                       \ncorrect=male     guess=female   name=Nigel                         \ncorrect=male     guess=female   name=Sascha                        \ncorrect=male     guess=female   name=Scottie                       \n"
     ]
    }
   ],
   "source": [
    "# 使用开发测试集执行错误分析，避免使用测试集进行错误分析和修正是为了避免对测试集的过拟合\n",
    "# 因为利用测试集的分析来增加规则，修正错误，则会使测试集的评估数据自然变好，但是测试集不能代表所有的数据特征\n",
    "# 因此利用训练集训练模型，使用开发测试集进行错误分析和规则修正，再利用测试集来评估，就可以尽可能保证数据的独立性和分布的均衡性\n",
    "errors = []\n",
    "for (name, tag) in devtest_names:\n",
    "    guess = classifier.classify(gender_features(name))\n",
    "    if guess != tag:\n",
    "        errors.append((tag, guess, name))\n",
    "for (tag, guess, name) in sorted(errors[:20]):\n",
    "    print('correct={:<8} guess={:<8s} name={:<30}'.format(tag, guess, name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.724\nNLTK-NB 分类器性能评估=  0.796\n--------------- >信息量大的特征< ---------------\nMost Informative Features\n                 suffix2 = 'na'           female : male   =     97.5 : 1.0\n                 suffix2 = 'la'           female : male   =     73.2 : 1.0\n                 suffix2 = 'ia'           female : male   =     39.6 : 1.0\n                 suffix2 = 'sa'           female : male   =     36.3 : 1.0\n                 suffix1 = 'a'            female : male   =     33.0 : 1.0\n--------------- >Neo's feature< ---------------\n{'suffix1': 'o', 'suffix2': 'eo'}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'suffix1': 'y', 'suffix2': 'ty'}\nTrinity is female\n"
     ]
    }
   ],
   "source": [
    "# 性别鉴定的两个特征：最后一个字母，最后两个字母，准确率更高（0.792）\n",
    "def gender_features(word):\n",
    "    return {'suffix1': word[-1:], 'suffix2': word[-2:]}\n",
    "\n",
    "\n",
    "gender_classifier(gender_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.74\nNLTK-NB 分类器性能评估=  0.82\n--------------- >信息量大的特征< ---------------\nMost Informative Features\n                 suffix2 = 'na'           female : male   =     97.5 : 1.0\n                 suffix2 = 'la'           female : male   =     73.2 : 1.0\n                 suffix2 = 'ia'           female : male   =     39.6 : 1.0\n                 suffix2 = 'sa'           female : male   =     36.3 : 1.0\n                 suffix1 = 'a'            female : male   =     33.0 : 1.0\n--------------- >Neo's feature< ---------------\n{'prefix': 'Ne', 'suffix1': 'o', 'suffix2': 'eo'}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'prefix': 'Tr', 'suffix1': 'y', 'suffix2': 'ty'}\nTrinity is female\n"
     ]
    }
   ],
   "source": [
    "# 性别鉴定的三个特征：头两个字母，最后一个字母，最后两个字母，准确率更高（0.81）\n",
    "def gender_features(word):\n",
    "    return {'prefix': word[0:2], 'suffix1': word[-1:], 'suffix2': word[-2:]}\n",
    "\n",
    "\n",
    "gender_classifier(gender_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.74\nNLTK-NB 分类器性能评估=  0.818\n--------------- >信息量大的特征< ---------------\nMost Informative Features\n                 suffix2 = 'na'           female : male   =     97.5 : 1.0\n                 suffix2 = 'la'           female : male   =     73.2 : 1.0\n                 suffix2 = 'ia'           female : male   =     39.6 : 1.0\n                 suffix2 = 'sa'           female : male   =     36.3 : 1.0\n                 suffix1 = 'a'            female : male   =     33.0 : 1.0\n--------------- >Neo's feature< ---------------\n{'prefix1': 'N', 'prefix2': 'Ne', 'suffix1': 'o', 'suffix2': 'eo'}\nNeo is male\n--------------- >Trinity's feature< ---------------\n{'prefix1': 'T', 'prefix2': 'Tr', 'suffix1': 'y', 'suffix2': 'ty'}\nTrinity is male\n"
     ]
    }
   ],
   "source": [
    "# 性别鉴定的四个特征：\n",
    "# 头一个字母，头两个字母，最后一个字母，最后两个字母，准确率更高（0.814）\n",
    "def gender_features(word):\n",
    "    return {'prefix1': word[0:1], 'prefix2': word[0:2], 'suffix1': word[-1:], 'suffix2': word[-2:]}\n",
    "\n",
    "\n",
    "gender_classifier(gender_features)"
   ]
  },
  {
   "source": [
    "### 1.3. 文档分类\n",
    "学习语料库中的标记，为新文档分配类别标签"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >categories()< ---------------\n['neg', 'pos']\n--------------- >fileids('neg')< ---------------\n['neg/cv000_29416.txt', 'neg/cv001_19502.txt', 'neg/cv002_17424.txt', 'neg/cv003_12683.txt', 'neg/cv004_12641.txt', 'neg/cv005_29357.txt', 'neg/cv006_17022.txt', 'neg/cv007_4992.txt', 'neg/cv008_29326.txt', 'neg/cv009_29417.txt', 'neg/cv010_29063.txt', 'neg/cv011_13044.txt', 'neg/cv012_29411.txt', 'neg/cv013_10494.txt', 'neg/cv014_15600.txt', 'neg/cv015_29356.txt', 'neg/cv016_4348.txt', 'neg/cv017_23487.txt', 'neg/cv018_21672.txt', 'neg/cv019_16117.txt']\n--------------- >words('/neg/cv995_23113.txt'< ---------------\n['if', 'anything', ',', '\"', 'stigmata', '\"', 'should', ...]\n"
     ]
    }
   ],
   "source": [
    "# 使用电影评论语料库，将每个评论归类为正面或者负面\n",
    "from nltk.corpus import movie_reviews\n",
    "\n",
    "show_subtitle(\"categories()\")\n",
    "print(movie_reviews.categories())\n",
    "show_subtitle(\"fileids('neg')\")\n",
    "print(movie_reviews.fileids('neg')[:20])\n",
    "show_subtitle(\"words('/neg/cv995_23113.txt'\")\n",
    "print(movie_reviews.words('neg/cv995_23113.txt'))\n",
    "\n",
    "documents = [\n",
    "        (list(movie_reviews.words(fileid)), category)\n",
    "        for category in movie_reviews.categories()\n",
    "        for fileid in movie_reviews.fileids(category)\n",
    "]\n",
    "random.shuffle(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >all_words.most_common(20)< ---------------\n[(',', 77717), ('the', 76529), ('.', 65876), ('a', 38106), ('and', 35576), ('of', 34123), ('to', 31937), (\"'\", 30585), ('is', 25195), ('in', 21822), ('s', 18513), ('\"', 17612), ('it', 16107), ('that', 15924), ('-', 15595), (')', 11781), ('(', 11664), ('as', 11378), ('with', 10792), ('for', 9961)]\n--------------- >word_features[:20]< ---------------\n[',', 'the', '.', 'a', 'and', 'of', 'to', \"'\", 'is', 'in', 's', '\"', 'it', 'that', '-', ')', '(', 'as', 'with', 'for']\ncontains(,) True\ncontains(the) True\ncontains(.) True\ncontains(a) True\ncontains(and) True\ncontains(of) True\ncontains(to) True\ncontains(') True\ncontains(is) True\ncontains(in) True\n"
     ]
    }
   ],
   "source": [
    "# Ex6-2 文档分类的特征提取器，其特征表示每个词是否在一个给定的文档中\n",
    "# 评论中使用的所有单词\n",
    "all_words = nltk.FreqDist(\n",
    "        w.lower()\n",
    "        for w in movie_reviews.words()\n",
    ")\n",
    "show_subtitle(\"all_words.most_common(20)\")\n",
    "print(all_words.most_common(20))\n",
    "# 取出部分单词作为特征\n",
    "# 任取2000个单词就可以产生很好的结果：0.81\n",
    "word_features_random = list(all_words)[:2000]\n",
    "show_subtitle(\"word_features[:20]\")\n",
    "print(word_features_random[:20])\n",
    "\n",
    "\n",
    "def document_features(document, word_features):\n",
    "    document_words = set(document)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['contains({})'.format(word)] = (word in document_words)\n",
    "    return features\n",
    "\n",
    "\n",
    "myDocument = movie_reviews.words('pos/cv957_8737.txt')\n",
    "myFeature = document_features(myDocument, word_features_random)\n",
    "for i, (key, value) in enumerate(myFeature.items()):\n",
    "    if i < 10:\n",
    "        print(key, value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.79\n",
      "NLTK-NB 分类器性能评估=  0.84\n",
      "--------------- >信息量大的特征< ---------------\n",
      "Most Informative Features\n",
      "   contains(outstanding) = True              pos : neg    =     13.1 : 1.0\n",
      "        contains(seagal) = True              neg : pos    =      7.9 : 1.0\n",
      "   contains(wonderfully) = True              pos : neg    =      7.5 : 1.0\n",
      "         contains(damon) = True              pos : neg    =      5.8 : 1.0\n",
      "          contains(lame) = True              neg : pos    =      5.7 : 1.0\n"
     ]
    }
   ],
   "source": [
    "# Ex6-3 训练和测试分类器以进行文档分类\n",
    "def document_classifier(word_features):\n",
    "    feature_sets = [\n",
    "            (document_features(d, word_features), c)\n",
    "            for (d, c) in documents\n",
    "    ]\n",
    "    train_set, test_set = feature_sets[100:], feature_sets[:100]\n",
    "\n",
    "    # 使用 Scikit-Learn 的 GaussNB 朴素贝叶斯 分类器进行分类\n",
    "    from sklearn.naive_bayes import GaussianNB\n",
    "    from nltk.classify.scikitlearn import SklearnClassifier\n",
    "\n",
    "    classifier = SklearnClassifier(GaussianNB(), sparse=False).train(train_set)\n",
    "    print(\"Scikit-GaussNB 分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "    \n",
    "    # 使用 NLTK 的 朴素贝叶斯 分类器进行分类训练\n",
    "    classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "    print(\"NLTK-NB 分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "\n",
    "    # 信息量大的特征（发现特征信息量比较小，说明这个特征效果不好）\n",
    "    show_subtitle(\"信息量大的特征\")\n",
    "    classifier.show_most_informative_features(5)\n",
    "    \n",
    "\n",
    "document_classifier(word_features_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.79\n",
      "NLTK-NB 分类器性能评估=  0.84\n",
      "--------------- >信息量大的特征< ---------------\n",
      "Most Informative Features\n",
      "   contains(outstanding) = True              pos : neg    =     13.1 : 1.0\n",
      "        contains(seagal) = True              neg : pos    =      7.9 : 1.0\n",
      "   contains(wonderfully) = True              pos : neg    =      7.5 : 1.0\n",
      "         contains(damon) = True              pos : neg    =      5.8 : 1.0\n",
      "          contains(lame) = True              neg : pos    =      5.7 : 1.0\n"
     ]
    }
   ],
   "source": [
    "# most_common() 返回的不是单词集合，不能作为正确的特征使用，需要将特征做进一步处理\n",
    "# word_features=list(all_words.most_common(2000))\n",
    "# 取出2000个高频单词产生更好的结果：0.82\n",
    "word_features_high_freq = [\n",
    "        word\n",
    "        for word, _ in all_words.most_common(2000)\n",
    "]\n",
    "document_classifier(word_features_high_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit-GaussNB 分类器性能评估=  0.58\n",
      "NLTK-NB 分类器性能评估=  0.58\n",
      "--------------- >信息量大的特征< ---------------\n",
      "Most Informative Features\n",
      "            contains(05) = True              neg : pos    =      1.7 : 1.0\n",
      "           contains(108) = True              neg : pos    =      1.7 : 1.0\n",
      "          contains(26th) = True              neg : pos    =      1.7 : 1.0\n",
      "            contains(32) = True              neg : pos    =      1.7 : 1.0\n",
      "            contains(39) = True              neg : pos    =      1.7 : 1.0\n"
     ]
    }
   ],
   "source": [
    "# 取出2000个低频单词产生更差的结果：0.69\n",
    "word_features_low_freq = [\n",
    "        word\n",
    "        for word, _ in all_words.most_common(20000)[-2000:]\n",
    "]\n",
    "document_classifier(word_features_low_freq)"
   ]
  },
  {
   "source": [
    "### 6.1.4 词性标注，词类标注，(Part-of-Speech Tagging, POST)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >common_suffixes[:20]< ---------------\n['e', ',', '.', 's', 'd', 't', 'he', 'n', 'a', 'of', 'the', 'y', 'r', 'to', 'in', 'f', 'o', 'ed', 'nd', 'is']\n"
     ]
    }
   ],
   "source": [
    "# 寻找最常见的单词后缀，可能也是最有信息量的单词后缀\n",
    "suffix_fdist = nltk.FreqDist()\n",
    "for word in brown.words():\n",
    "    word = word.lower()\n",
    "    suffix_fdist[word[-1:]] += 1\n",
    "    suffix_fdist[word[-2:]] += 1\n",
    "    suffix_fdist[word[-3:]] += 1\n",
    "common_suffixes = [\n",
    "        suffix\n",
    "        for (suffix, count) in suffix_fdist.most_common(100)\n",
    "]\n",
    "show_subtitle(\"common_suffixes[:20]\")\n",
    "print(common_suffixes[:20])"
   ]
  },
  {
   "source": [
    "# 定义特征提取函数，以前100个单词后缀为特征，确定每个单词受这100个后缀的影响\n",
    "# 例如：should的后缀'ld'存在，后缀'en'不存在\n",
    "def pos_features(word):\n",
    "    features = {}\n",
    "    for suffix in common_suffixes:\n",
    "        features['endswith({})'.format(suffix)] = word.lower().endswith(suffix)\n",
    "    return features"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 34,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 建立特征集合，每个单词的特征（后缀'en':存在）+标注（NN)为一条数据，组成的特征集合\n",
    "tagged_words = brown.tagged_words(categories='news')\n",
    "feature_sets = [(pos_features(n), g) for (n, g) in tagged_words]\n",
    "\n",
    "size = int(len(feature_sets) * 0.1)\n",
    "train_set, test_set = feature_sets[size:], feature_sets[:size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "('The', 'AT')"
      ]
     },
     "metadata": {},
     "execution_count": 41
    }
   ],
   "source": [
    "tagged_words[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "NLTK 决策树分类器性能评估=  0.6270512182993535\ncats 标注的结果： NNS\nif endswith(the) == False: \n  if endswith(,) == False: \n    if endswith(s) == False: \n      if endswith(.) == False: return '.'\n      if endswith(.) == True: return '.'\n    if endswith(s) == True: \n      if endswith(is) == False: return 'PP$'\n      if endswith(is) == True: return 'BEZ'\n  if endswith(,) == True: return ','\nif endswith(the) == True: return 'AT'\n\n"
     ]
    }
   ],
   "source": [
    "# 下面这行代码执行时间过长，可以尝试减少 word_features 中的特征数量\n",
    "# i5-5200U 需要运行10分钟以上！！！\n",
    "# i7-9700 需要的时间也很长\n",
    "# NLTK决策树模型的优点：容易解释，甚至可以使用伪代码的形式输出\n",
    "# NLTK 决策树模型的缺点：运行速度慢，精度还不高\n",
    "classifier = nltk.DecisionTreeClassifier.train(train_set)\n",
    "print(\"NLTK 决策树分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "print(\"cats 标注的结果：\",classifier.classify(pos_features('cats')))\n",
    "print(classifier.pseudocode(depth=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Scikit 决策树分类器性能评估=  0.6459472899055196\ncats 标注的结果： NNS\n"
     ]
    }
   ],
   "source": [
    "# 使用 Scikit-Learn 运行速度会快很多，但是不能输出伪代码\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "classifier = SklearnClassifier(DecisionTreeClassifier()).train(train_set)\n",
    "print(\"Scikit 决策树分类器性能评估= \", nltk.classify.accuracy(classifier, test_set))\n",
    "print(\"cats 标注的结果：\",classifier.classify(pos_features('cats')))"
   ]
  },
  {
   "source": [
    "### 6.1.5 基于上下文语境特征进行词性标注\n",
    "\n",
    "-   通过上下文提高分类的精度，例如：large后面的可能是名词；\n",
    "-   但是不能根据前词的标记判断下个词的类别，例如：前面是形容词，后面的可能是名词\n",
    "-   下节的序列分类就是使用联合分类器模型，为一些相关的输入选择适当的标签"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex6-4 词性分类器，特征检测器通过一个单词的上下文来决定这个单词的词性标记\n",
    "def pos_features(sentence: list, i: int) -> dict:\n",
    "    \"\"\"更加复杂的特征提取函数\n",
    "    :param sentence:用于提取单词的句子，提供句子才可以提供上下文语境特征\n",
    "    :type sentence:list\n",
    "    :param i:句子中需要提取特征的单词的位置\n",
    "    :type i:int\n",
    "    :return:相应单词的特征集\n",
    "    :rtype:dict \"\"\"\n",
    "    # 当前词的后缀特征\n",
    "    features = {\n",
    "        'Suffix(1)': sentence[i][-1:],\n",
    "        'Suffix(2)': sentence[i][-2:],\n",
    "        'Suffix(3)': sentence[i][-3:]\n",
    "    }\n",
    "    # 单词前面的词作为特征，如果单词为头一个单词就设置前面一个单词为<START>\n",
    "    if i == 0:\n",
    "        features['prev-word'] = '<START>'\n",
    "    else:\n",
    "        features['prev-word'] = sentence[i - 1]\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0.7891596220785678"
      ]
     },
     "metadata": {},
     "execution_count": 47
    }
   ],
   "source": [
    "# (brown.sents()[0],8) == 'investigation'\n",
    "type(pos_features(brown.sents()[0], 8))\n",
    "tagged_sents = brown.tagged_sents(categories='news')\n",
    "feature_sets = []\n",
    "for tagged_sent in tagged_sents:\n",
    "    untagged_sent = nltk.tag.untag(tagged_sent)\n",
    "    for i, (word, tag) in enumerate(tagged_sent):\n",
    "        # 特征集合中的元素必须是tuple的形式\n",
    "        feature_sets.append((pos_features(untagged_sent, i), tag))  \n",
    "\n",
    "size = int(len(feature_sets) * 0.1)\n",
    "train_set, test_set = feature_sets[size:], feature_sets[:size]\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "print(\"基于上下文语境特征进行词性标注的模型精度= \", nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "source": [
    "### 6.1.6 基于序列分类的词性标注（通过上下文的标签提高分类的精度）\n",
    "为了获取相关分类任务之间的依赖关系，可以使用联合分类器模型，使用连续分类或者贪婪序列分类的序列分类器策略，得到更高的分类准确率"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ex6-5 使用连贯分类器进行词性标注\n",
    "# 1. 特征提取器：提供具有上下文描述的特征\n",
    "def pos_features(sentence: list, i: int, history: list) -> dict:\n",
    "    \"\"\"\n",
    "        >>> pos_features(['I', 'love', 'you','.'], 1, ['N'])\n",
    "        {'Suffix(1)': 'e', 'Suffix(2)': 've', 'Suffix(3)': 'ove', 'prev-word': 'I', 'prev-tag': 'N'}\n",
    "    :param sentence:用于提取单词的句子，提供句子才可以提供上下文语境特征\n",
    "    :type sentence:list\n",
    "    :param i:句子中需要提取特征的单词的位置\n",
    "    :type i:int\n",
    "    :param history:history中的每个标记对应sentence中的一个词，但是history只包括已经归类的词的标记\n",
    "    :type history:\n",
    "    :return:相应单词的特征集\n",
    "    :rtype:dict\n",
    "    \"\"\"\n",
    "    features = {\n",
    "        'Suffix(1)': sentence[i][-1:],\n",
    "        'Suffix(2)': sentence[i][-2:],\n",
    "        'Suffix(3)': sentence[i][-3:]\n",
    "    }\n",
    "    if i == 0:\n",
    "        features['prev-word'] = '<START>'\n",
    "        features['prev-tag'] = '<START>'\n",
    "    else:\n",
    "        features['prev-word'] = sentence[i - 1]\n",
    "        features['prev-tag'] = history[i - 1]\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'Suffix(1)': 'n',\n",
       " 'Suffix(2)': 'on',\n",
       " 'Suffix(3)': 'ton',\n",
       " 'prev-word': 'The',\n",
       " 'prev-tag': 'N'}"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "history = ['N']\n",
    "pos_features(brown.sents()[0], 1, history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. 基于序列分类的词性标注\n",
    "class ConsecutivePosTagger(nltk.TaggerI):\n",
    "    def __init__(self, train_sents):\n",
    "        train_set = []\n",
    "        for tagged_sent in train_sents:\n",
    "            untagged_sent = nltk.tag.untag(tagged_sent)\n",
    "            # 将（单词，标签）元组组成的列表中的标签去掉，只保留单词组成的列表\n",
    "            # print(untagged_sent)  \n",
    "            \n",
    "            history = []\n",
    "            for i, (word, tag) in enumerate(tagged_sent):\n",
    "                featuresets = pos_features(untagged_sent, i, history)\n",
    "                train_set.append((featuresets, tag))\n",
    "                history.append(tag)\n",
    "        self.classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "\n",
    "    def tag(self, sentence):\n",
    "        history = []\n",
    "        for i, word in enumerate(sentence):\n",
    "            featuresets = pos_features(sentence, i, history)\n",
    "            tag = self.classifier.classify(featuresets)\n",
    "            history.append(tag)\n",
    "        return zip(sentence, history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "train_sents[0]=  [('He', 'PPS'), ('assured', 'VBD'), ('Mr.', 'NP'), ('Martinelli', 'NP'), ('and', 'CC'), ('the', 'AT'), ('council', 'NN'), ('that', 'CS'), ('he', 'PPS'), ('would', 'MD'), ('study', 'VB'), ('the', 'AT'), ('correct', 'JJ'), ('method', 'NN'), ('and', 'CC'), ('report', 'VB'), ('back', 'RB'), ('to', 'IN'), ('the', 'AT'), ('council', 'NN'), ('as', 'QL'), ('soon', 'RB'), ('as', 'CS'), ('possible', 'JJ'), ('.', '.')]\n"
     ]
    }
   ],
   "source": [
    "tagged_sents = brown.tagged_sents(categories='news')\n",
    "size = int(len(tagged_sents) * 0.1)\n",
    "train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]\n",
    "print(\"train_sents[0]= \", train_sents[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0.7980528511821975\n"
     ]
    }
   ],
   "source": [
    "tagger = ConsecutivePosTagger(train_sents)\n",
    "print(\"基于序列分类的词性标注模型的精度= \", tagger.evaluate(test_sents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[('The', 'AT'), ('Fulton', 'NP-TL'), ('County', 'NN-TL'), ('Grand', 'JJ-TL'), ('Jury', 'NN-TL'), ('said', 'VBD'), ('Friday', 'NR'), ('an', 'AT'), ('investigation', 'NN'), ('of', 'IN'), (\"Atlanta's\", 'NP$'), ('recent', 'JJ'), ('primary', 'NN'), ('election', 'NN'), ('produced', 'VBD'), ('``', '``'), ('no', 'AT'), ('evidence', 'NN'), (\"''\", \"''\"), ('that', 'CS'), ('any', 'DTI'), ('irregularities', 'NNS'), ('took', 'VBD'), ('place', 'NN'), ('.', '.')]\n(('The', 'AT'), '``')\n(('Fulton', 'NP-TL'), 'AT-TL')\n(('County', 'NN-TL'), 'NNS-TL')\n(('Grand', 'JJ-TL'), 'IN-TL')\n(('Jury', 'NN-TL'), 'AT-TL')\n(('said', 'VBD'), 'NNS-TL')\n(('Friday', 'NR'), 'IN-TL')\n(('an', 'AT'), 'AT-TL')\n(('investigation', 'NN'), 'NNS-TL')\n(('of', 'IN'), 'IN-TL')\n((\"Atlanta's\", 'NP$'), 'AT-TL')\n(('recent', 'JJ'), 'NNS-TL')\n(('primary', 'NN'), 'IN-TL')\n(('election', 'NN'), 'AT-TL')\n(('produced', 'VBD'), 'NNS-TL')\n(('``', '``'), 'IN-TL')\n(('no', 'AT'), 'AT-TL')\n(('evidence', 'NN'), 'NNS-TL')\n((\"''\", \"''\"), 'IN-TL')\n(('that', 'CS'), 'AT-TL')\n(('any', 'DTI'), 'NNS-TL')\n(('irregularities', 'NNS'), 'IN-TL')\n(('took', 'VBD'), 'AT-TL')\n(('place', 'NN'), 'NNS-TL')\n(('.', '.'), 'IN-TL')\n"
     ]
    }
   ],
   "source": [
    "print(test_sents[0])\n",
    "\n",
    "for label in tagger.tag(test_sents[0]):\n",
    "    print(label)"
   ]
  },
  {
   "source": [
    "### 6.1.7 序列分类器中的其他方法\n",
    "1.  基于转换的联合分类：Ref：Sec5.6节描述的Brill标注器，解决了前面分类器一旦分类就无法改变的问题\n",
    "2.  基于HMM的联合分类：\n",
    "      -   特点：不仅考虑了单词的上下文，还可以考虑更长的依赖性\n",
    "      -   输出：不是某个单词标记的最大可能性，而是整个序列中所有单词标记的最大可能性\n",
    "      -   常用的模型：最大熵马尔可夫模型 和 线性链条件随机场模型"
   ],
   "cell_type": "markdown",
   "metadata": {}
  }
 ]
}